package com.spider.ask120.webmagic;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.jdiy.core.Args;
import org.jdiy.core.Dao;
import org.jdiy.core.Rs;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import com.spider.utils.HttpClientUtil;
import com.spider.utils.JdiyUtils;
import com.spider.utils.PinyinUtil;


/**
 * 描        述：爬数据-健康咨询(频道)
 * 创建时间：2016-8-15
 * @author Jibaole
 */
public class PinDao extends JdiyUtils implements PageProcessor {
	   public static final String URL_INDEX = "http://m.120ask.com/healthart/wx_art_home_all/";
	   public static final String URL_LIST = "/?kswys_wx=1&from=atr_home_all";
	   public static final String URL_tag_list = "/healthart/wx_lists/\\?tid=[0-9]+";
	   public static final String URL_tag_list_cont = "/cron/wxartlist/jump";
	   int num=0;
	   List<String> change_Ids = new ArrayList<String>();//频道id
	   List<String> tag_Ids = new ArrayList<String>();//标签id
	   List<String> cont_idS = new ArrayList<String>();//内容id
	  
	
	// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
	private Site site = Site.me().setRetryTimes(3).setSleepTime(1);
	public Site getSite() {  
		return site;
	}
	

	 @Override
	    public void process(Page page) {
		try {
			Dao dao = getDao();
		
	        //列表页
	        if (page.getUrl().regex(URL_INDEX+"$").match()) {
	        	System.err.println("#########################>>>频道页<<<<###################################");
	        	//获取频道列表URL 
	        	List<String> list = page.getHtml().xpath("//ul[@class='ChanelList']/a[@href]/@href").all(); 
	        	List<String> title = page.getHtml().xpath("//ul[@class='ChanelList']/a/li/p/b/text()").all();
	        	List<String> jianjie = page.getHtml().xpath("//ul[@class='ChanelList']/a/li/p/span/text()").all();
	        	//List<String> img =  page.getHtml().xpath("//ul[@class='ChanelList']/a/li/img/@src").all();
					list.remove(2);//移除健康饮食
		        	list.remove(2);//移除民间偏方
		        	list.remove(3);//移除健康经验
		        	
		        	title.remove(2);//移除健康饮食
		        	title.remove(2);//移除民间偏方
		        	title.remove(3);//移除健康经验
		        	
		        	jianjie.remove(2);//移除健康饮食
		        	jianjie.remove(2);//移除民间偏方
		        	jianjie.remove(3);//移除健康经验
	        	if(list != null){
	        		for(int i=0;i<list.size();i++){
	        			Rs channel= new Rs("channel");
	        			String linkUrl=list.get(i);
	        			//String img_str=img.get(i);
	        			channel.set("name", title.get(i));
	        			channel.set("pinyin", PinyinUtil.converterToFirstSpell(title.get(i)));
	        			channel.set("abst", jianjie.get(i));
	        			channel.set("str_id", linkUrl.substring(linkUrl.indexOf("/wx")+1, linkUrl.indexOf("/?")));
	        			channel.set("level", 1);
	        			dao.save(channel);
	        			change_Ids.add(channel.get("id"));
	        		}
	        	}
	        	/*List<String> list1= new ArrayList<String>();
	        	list1.add(list.get(0));*/
	        	page.addTargetRequests(list);
	            //tag 标签
	        } else if(page.getUrl().regex(URL_LIST+"$").match()) {
	        	System.err.println("#########################>>>标签页<<<<###################################");
	        	String change_Id =change_Ids.get(num);
	        	++num;
	        	//得到所有标签
	        	List<String> tagList = page.getHtml().xpath("//div[@class='ChanelHomeBox']/ul/li/a/text()").all();
	        	List<String> tag_url = page.getHtml().xpath("//div[@class='ChanelHomeBox']/ul/li/a/@href").all();
	        	for(int i=0;i<tagList.size();i++){
	        		String linkUrl=tag_url.get(i);
	        		Rs channel_tag= new Rs("channel");
	        		channel_tag.set("name", tagList.get(i));
	        		channel_tag.set("pinyin", PinyinUtil.converterToFirstSpell(tagList.get(i)));
        			channel_tag.set("parent_id", change_Id);
        			if(linkUrl.indexOf("tid") > 0){
        				channel_tag.set("str_id", linkUrl.substring(linkUrl.indexOf("=")+1, linkUrl.indexOf("&")));
        			}else{
        				tag_url.remove(i);//移除"怀孕40周"
        				tagList.remove(i);
        			}
        			channel_tag.set("level", 2);
        			if(linkUrl.indexOf("tid") > 0){//移除地址、内容不规则的资源
        			dao.save(channel_tag);
        			tag_Ids.add(channel_tag.get("id"));
        			}
	        	}
	        	/*List<String> list1= new ArrayList<String>();
	        	list1.add(tag_url.get(0));*/
	        	num = 0;
	        	page.addTargetRequests(tag_url);
	        //tag下，列表页
	        }else if(page.getUrl().regex(URL_tag_list).match()){
	        	System.err.println("#########################>>>详细内容，列表页<<<<###################################");
	        	String tag_Id =tag_Ids.get(num);
	        	++num;
	        	List<String> urlList = new ArrayList<String>();
	        	String url = page.getUrl().toString();
	        	String tid=url.substring(url.indexOf("=")+1, url.indexOf("&"));
					//无下级栏目，开始加载分页数据列表，获取正文URL
					Map<String, String> params = new HashMap<String, String>();
					params.put("tid",tid);
					params.put("tj_type", "5");
					params.put("order", "1");
					int pageIndex = 0;	
					while(true){ 
						pageIndex ++;
						params.put("page", pageIndex+"");
						//System.out.println("第"+pageIndex+"页");
						String listJson = HttpClientUtil.getInstance().sendHttpPost("http://m.120ask.com/healthart/wx_art_more/", params);
						if(!StringUtils.isEmpty(listJson)){
								try {
									Document doc = Jsoup.parse(listJson);
									//遍历标签
									for (Element link : doc.getElementsByAttribute("href")) {
									 String linkUrl = link.attr("href");
									 Rs cont= new Rs("article");
									 cont.set("channel_id", tag_Id);
									 cont.set("title", link.select("dd").get(0).text());
									 Rs campRs = dao.rs(new Args("channel", "id = "+tag_Id));
									 if(!campRs.isNull()){
										 cont.set("source", campRs.get("name")); 
									 }
									 cont.set("pic", link.select("img").attr("src"));
									 cont.set("abst", link.select("dd").get(1).text());
									 dao.save(cont);
									 cont_idS.add(cont.get("id"));
									 urlList.add(linkUrl);
									}
							} catch (Exception e) { 
								e.printStackTrace();
							}
						}else{
							break;
						}
						num = 0;
						page.addTargetRequests(urlList);
					}
	        //详细内容页
	        }else if(page.getUrl().toString().indexOf(URL_tag_list_cont) > 0){
	        	System.err.println("#########################>>>详细内容页<<<<###################################");
	        	String linkUrl = page.getUrl().toString();
	        	String tid=linkUrl.substring(linkUrl.indexOf("=")+1, linkUrl.indexOf("&"));
	        	 Rs cont = dao.rs(new Args("article", "id = "+cont_idS.get(num)));
	        	 ++num;
	        	 cont.set("str_id",tid);
	        	cont.set("content",page.getHtml().xpath("//div[@class='article_cont-divp']/html()").toString());
	        	cont.set("create_time",page.getHtml().xpath("//dd[@class='article_cont-dd2']/span[1]/text()").toString());
	        	String  browse_count = page.getHtml().xpath("//dd[@class='article_cont-dd2']/span[2]").toString();
	        	if(!StringUtils.isEmpty(browse_count)){
	        		if(browse_count.indexOf("(") >0 && browse_count.indexOf(")") >0){
	        			cont.set("browse_count", browse_count.substring(browse_count.indexOf("(")+1, browse_count.indexOf(")")));	
	        		}
	        	}
	        	dao.save(cont);
	        }
		} catch (Exception e) {
			e.printStackTrace();
		}
	 }
	 
	public static void main(String[] args) {
		Spider.create(new PinDao()).addUrl("http://m.120ask.com/healthart/wx_art_home_all/").thread(1).run(); 
    }
 
}
